# encoding=utf-8
import cPickle as pickle
import json
import re
import time

import httplib2
from scrapy import Selector
from scrapy.http import Request
from scrapy.spiders import CrawlSpider

from jobspider.items import JobItem, CompanyInfoItem, JobDetailItem


class Spider(CrawlSpider):
    name = "job51Spider"

    def start_requests(self):
        page = 1

        # 城市编码
        # gzhou
        codestr = "030200"

        keyword = "java"

        base_url = "http://search.51job.com/jobsearch/search_result.php?fromJs=1&funtype=0000&industrytype=00&curr_page=%s"

        if keyword:
            base_url += "&keyword=" + keyword

        if codestr:
            base_url += "&jobarea=" + codestr

        search_url = base_url % (page)

        print "search_url", search_url
        yield Request(url=search_url, callback=self.parse_search, meta={"page": str(page), "base_url": base_url})

    def parse_search(self, response):
        # print response.body.decode('gbk').encode('utf-8')
        content = response.body.decode('gbk').encode("utf-8")
        sel = Selector(text=content)

        page = int(response.meta["page"])
        now = time.strftime("%Y-%m-%d", time.localtime())
        sdiv = sel.xpath("//div[@class='el' and not(@id)]")
        isturnpage = True

        for div in sdiv:
            job_name = div.xpath("p/span/a/@title").extract_first().strip()
            job_url = div.xpath("p/span/a/@href").extract_first().strip()

            span = div.xpath("span")
            company_name = span[0].xpath("a/@title").extract_first().strip()
            company_detail_url = span[0].xpath("a/@href").extract_first().strip()

            job_detail_address = span[1].xpath("text()").extract_first().strip()
            city_district = job_detail_address.split("-")
            city = city_district[0]
            district = None
            if len(city_district) > 1:
                district = city_district[1]

            salary = None
            publish_time = None
            try:
                salary = span[2].xpath("text()").extract_first().strip()
                publish_time = "2017-" + span[3].xpath("text()").extract_first().strip()

                print publish_time, now
                # 不用翻页 停止爬取
                if publish_time != now:
                    # print publish_time + " stop!!!"
                    isturnpage = False
                    break

            except:
                print "salary = None"

            query_time = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())

            job = JobItem()

            job["job_url"] = job_url
            job["job_name"] = job_name

            job["company_name"] = company_name
            job["company_detail_url"] = company_detail_url

            companyinfo = CompanyInfoItem()
            companyinfo["company_name"] = company_name
            companyinfo["company_detail_url"] = company_detail_url

            job["city"] = city

            job["district"] = district

            job["job_detail_address"] = job_detail_address

            job["salary"] = salary

            job["publish_time"] = publish_time

            job["query_time"] = query_time
            job["query_type"] = "前程无忧"

            yield job
            print "yield job"
            yield Request(url=job_url, callback=self.parse_job_detail,
                          meta={"job_url": job_url, "companyinfo": companyinfo})

        # if page == 1:
        #     total_number = re.findall("共.*?页", content)[0]
        #     total_number = total_number[3:len(total_number) - 3]
        #     base_url = response.meta["base_url"]
        #     print "total_number", total_number
        #     for page in range(2, int(total_number) + 1, 1):
        #         print "page", page
        #
        #         search_url = base_url % ("030200", page)
        #         yield Request(url=base_url, callback=self.parse_search,
        #                       meta={"page": str(page), "base_url": base_url})

        if isturnpage:

            total_number = re.findall("共.*?页", content)[0]
            total_number = total_number[3:len(total_number) - 3]
            if page < total_number:
                page += 1
                print "turnpage!!!", page
                base_url = response.meta["base_url"]
                search_url = base_url % (page)
                yield Request(url=search_url, callback=self.parse_search,
                              meta={"page": str(page), "base_url": base_url})

    def parse_job_detail(self, response):

        content = response.body.decode('gbk').encode("utf-8")
        sel = Selector(text=content)

        job_url = response.meta["job_url"]
        companyinfo = response.meta["companyinfo"]

        cn = sel.xpath("//p[@class='msg ltype']/text()").extract_first()
        if cn:
            cn = cn.strip()
            cn_msg = cn.split("|")
        try:
            conpany_type = cn_msg[0]
            conpany_size = cn_msg[1]
            company_orientation = cn_msg[2]
        except:
            conpany_type = None
            conpany_size = None
            company_orientation = None

        sp4 = sel.xpath("//span[@class='sp4']")
        sp4_em = {}
        for em in sp4:
            em_id = em.xpath("em/@class").extract_first()
            sp4_em[em_id] = em.xpath("text()").extract_first()
        if "i1" in sp4_em:
            experience = sp4_em["i1"]
        else:
            experience = None
        if "i2" in sp4_em:
            education_level = sp4_em["i2"]
        else:
            education_level = None
        if "i3" in sp4_em:
            recruit_num = sp4_em["i3"]
        else:
            recruit_num = None
        welfare = sel.xpath("//p[@class='t2']/span/text()").extract()

        work_welfare = ''.join(a + "," for a in welfare)[:-1]

        occupation_info = sel.xpath("//div[@class='bmsg job_msg inbox']").extract_first()
        company_info = sel.xpath("//div[@class='tmsg inbox']").extract_first()

        job_contact_information = sel.xpath("//p[@class='fp']/text()").extract()
        if len(job_contact_information) > 1:
            job_contact_information = job_contact_information[1]

        query_time = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())

        jobdetail = JobDetailItem()
        jobdetail["job_url"] = job_url
        # 工作年限
        jobdetail["experience"] = experience
        # 教育水平
        jobdetail["education_level"] = education_level
        # 招聘人数
        jobdetail["recruit_num"] = recruit_num
        # 公司福利
        jobdetail["work_welfare"] = work_welfare
        # 职位信息
        jobdetail["occupation_info"] = occupation_info
        # 工作联系方式
        jobdetail["job_contact_information"] = job_contact_information
        # 查询时间
        jobdetail["query_time"] = query_time

        # 公司信息
        companyinfo["company_info"] = company_info
        # 公司类型
        companyinfo["conpany_type"] = conpany_type
        # 公司人数
        companyinfo["conpany_size"] = conpany_size
        # 公司定位
        companyinfo["company_orientation"] = company_orientation

        yield companyinfo
        yield jobdetail
